from selenium import webdriver 
from selenium.webdriver.chrome.options import Options 
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By 
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC 
from scrapy.http import HtmlResponse
import time,random
#import user_agent
class SpiderDownloaderMiddleware(object):
    def __init__(self):
        self.USER_AGENTS = [
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
        "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
        "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
        "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
        "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
        ]
        
    def process_request(self,request,spider):
        if('type' not in request.meta.keys()):
            chrome_options = Options() 
            chrome_options.add_argument('--headless') 
            chrome_options.add_argument('user-agent="'+self.USER_AGENTS[random.randint(0,9)]+'"')
            chromedriver="C:/Users/孙运武/AppData/Local/Google/Chrome/Application/chromedriver.exe"
            driver = webdriver.Chrome(chromedriver,chrome_options=chrome_options) 
            time.sleep(random.random()*3)
            driver.get(request.url)
            try:
                element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "main")))
                driver.save_screenshot("state.png")
                return HtmlResponse(url=request.url,body=driver.page_source,request=request,encoding="utf-8",status=200)
            except TimeoutException:
                return HtmlResponse(url=request.url, status=500, request=request)






